keep = ls()

########################
#Prepare Newspaper data#
########################

#Get list of newspaper issues from NP.com
issues_1 = fread('./data/wisconsin_newspapers/raw/wi_issues.csv') %>% 
  .[, list(publication = pubTitle,
           issue_date = as.Date(date))]
#Get list of newspaper issues from other sources
issues_2 = fread('./data/wisconsin_newspapers/raw/extra_wi_issues.csv') %>%
  .[, list(publication = paper,
           issue_date = as.Date(issue_date, "%m/%d/%Y"))]
#All issues
issues = rbind(issues_1, issues_2)

#Add newspaper ids/partisanship to issue list
paper_list = fread("./data/wisconsin_newspapers/raw/paper_affiliation.csv") %>%
  .[, list(publication = paper,
           county, 
           partisanship,
           paper_id)]

#Merge to issues
setkey(paper_list, publication)
setkey(issues, publication)
issues = paper_list[issues]

#Get unique issue ID
issues[, issue_id := 1:.N]

#Load Newspaper Article Classifications:

articles = fread('./data/wisconsin_newspapers/raw/wisconsin_article_coding.csv') %>%
  .[, list(Timestamp,
           url,
           suffrage_position = `What position does this article take on suffrage?`,
           pro_suffrage_raw = `Which of the following are used to make Pro-suffrage arguments?`,
           anti_suffrage_raw = `Which of the following are used to make Anti-suffrage arguments?`,
           other_position_raw = `Does this article contain any of the following?`,
           publication = `Name of newspaper`,
           issue_date = as.Date(issue_date, "%m/%d/%Y")
  )]

#Merge in newspaper ids to articles
setkey(articles, publication)
setkey(paper_list, publication)
articles = paper_list[articles] %>% .[!is.na(paper_id)]

issue_ids = issues[, list(paper_id, issue_date, issue_id)]
setkey(issue_ids, paper_id, issue_date)
setkey(articles, paper_id, issue_date)
articles = issue_ids[articles]

#Recode newspaper article content:

#Overall position on suffrage
articles[, suffrage_pro := suffrage_position %in% "Pro suffrage"]
articles[, suffrage_anti := suffrage_position %in% "Anti suffrage"]

#mentions of suffrage-adjacent issues
articles[, suffrage_republican_issue := str_detect(other_position_raw, "Claim that loyal Republicans/Union party members must support suffrage")]
articles[, suffrage_republican_activist := str_detect(other_position_raw, "Republicans/Union party members attacking members of their own party for NOT supporting suffrage")]
articles[, suffrage_link_fighting := str_detect(other_position_raw, "Link voting for suffrage to fighting the rebels")]
articles[, suffrage_republicans_oppose := str_detect(other_position_raw, "Republicans/Union party members opposing suffrage")]
articles[, suffrage_republicans_personal_support := str_detect(other_position_raw, "Republicans/Union party members personally support suffrage")]
articles[, suffrage_republicans_personal_position := str_detect(other_position_raw, "Demand for Republican/Union party members to state a position on suffrage")]


#Pro Suffrage Arguments
articles[, suffrage_pro_black_loyalty := str_detect(pro_suffrage_raw, "African American people loyal in the war and earned citizenship/voting rights")]
articles[, suffrage_pro_usct_loyalty := str_detect(pro_suffrage_raw, "African Americans soldiers loyal in the war and earned citizenship/voting rights")]
articles[, suffrage_pro_needed_end_slavery := str_detect(pro_suffrage_raw, "African American voters needed to ensure end to slavery")]
articles[, suffrage_pro_needed_win_peace := str_detect(pro_suffrage_raw, "African American voters needed to ensure victories of the war remain won")]
articles[, suffrage_pro_soldier_endorse := str_detect(pro_suffrage_raw, "Former soldiers/officers endorse suffrage")]
articles[, suffrage_pro_needed_anti_rebel := str_detect(pro_suffrage_raw, "Need black voters to prevent return of rebels to power")]
articles[, suffrage_pro_needed_loyal := str_detect(pro_suffrage_raw, "Need loyal voters in the South")]
articles[, suffrage_pro_anti_copperhead := str_detect(pro_suffrage_raw, "Copperheads")]
articles[, suffrage_pro_punishment := str_detect(pro_suffrage_raw, "Punish")]
articles[, suffrage_pro_representation := str_detect(pro_suffrage_raw, "unfair advantage")]

#Pro Suffrage Categories
articles[, suffrage_pro_earned := suffrage_pro_black_loyalty | suffrage_pro_usct_loyalty]
articles[, suffrage_pro_war_aims := suffrage_pro_needed_end_slavery | suffrage_pro_needed_win_peace | suffrage_pro_needed_loyal]
articles[, suffrage_pro_treason := suffrage_pro_anti_copperhead | suffrage_pro_punishment]
articles[, suffrage_pro_strategic := suffrage_pro_needed_anti_rebel | suffrage_pro_representation]

#Anti Suffrage Arguments
articles[, suffrage_anti_soldier_oppose := str_detect(anti_suffrage_raw, "Former soldiers/officers oppose suffrage")]
articles[, suffrage_anti_partisan := str_detect(anti_suffrage_raw, "Suffrage is insincere partisan ploy")]
articles[, suffrage_anti_social_equality := str_detect(anti_suffrage_raw, "social equality")]
articles[, suffrage_anti_abolitionist := str_detect(anti_suffrage_raw, "abolitionists")]
articles[, suffrage_anti_white_rights := str_detect(anti_suffrage_raw, "Suffrage tramples on the rights of white men")]
articles[, suffrage_anti_alienate_whites := str_detect(anti_suffrage_raw, "Suffrage will alienate white voters from Republican party")]
articles[, suffrage_anti_anger_south := str_detect(anti_suffrage_raw, "Suffrage will antagonize white Southerners")]
articles[, suffrage_anti_black_supremacy := str_detect(anti_suffrage_raw, "black supremacy")]
articles[, suffrage_anti_only_union := str_detect(anti_suffrage_raw, "only Union")]



########################################
#Summarize Republican Newspaper Content#
########################################

#Sum article content by issue: 
issue_content = articles[, 
                list(pro = sum(suffrage_pro)/.N,
                     anti = sum(suffrage_anti)/.N,
                     n = .N
                ),
                by = list(paper_id, issue_id)]

setkey(issue_content, issue_id)
setkey(issues, issue_id)
issue_content = issue_content[issues]

issue_content[is.na(pro), pro := 0]
issue_content[is.na(anti), anti := 0]
issue_content[is.na(n), n := 0]

#Fraction suffrage-related coverage that was "pro" suffrage, by newspaper
issue_content[partisanship %in% "republican", sum(n*pro) / sum(n*pro + n*anti), by = i.paper_id] %>% 
  .$V1 %>% hist(breaks = seq(0,1,0.05), 
                main = "Suffrage Coverage\nAcross Republican Papers",
                xlab = "Fraction Pro-Suffrage")

cat("Fraction of Republican newspapers with > 75% of coverage in favor of suffrage: \n")
issue_content[partisanship %in% "republican", sum(n*pro) / sum(n*pro + n*anti), by = i.paper_id] %>% 
  .$V1 %>%
  `>` (0.75) %>%
  mean


cat("Fraction of Republican Newspapers arguing for suffrage on basis of African American loyalty: \n")
articles[partisanship %in% "republican", sum(suffrage_pro_earned), by = paper_id] %>% 
  .$V1 %>% 
  `>` (0) %>%
  mean


cat("Mean Fraction of Pro-Suffrage Coverage in Republican Newspapers arguing for suffrage on basis of African American loyalty: \n")
articles[partisanship %in% "republican", sum(suffrage_pro_earned)/sum(suffrage_pro), by = paper_id] %>% 
  .$V1 %>% 
  mean

cat("Fraction of Republican Newspapers arguing for suffrage on basis of Achieving war aims of ending slavery, winning the peace, ensuring loyalty \n")
articles[partisanship %in% "republican", any(suffrage_pro_war_aims), by = paper_id] %>% .$V1 %>% mean


cat("Fraction of Republican Newspapers arguing for suffrage on basis of Preventing return of rebels to power/offsetting unfair representation of Southerners \n")
articles[partisanship %in% "republican", any(suffrage_pro_strategic), by = paper_id] %>% .$V1 %>% mean


cat("Mean Fraction of Pro-Suffrage Coverage in Republican Newspapers that endorse suffrage on basis of Preventing return of rebels to power/offsetting unfair representation of Southerners\n")
articles[partisanship %in% "republican", sum(suffrage_pro_strategic)/sum(suffrage_pro), by = paper_id] %>% 
  .$V1 %>% mean

cat("Mean Fraction of Pro-Suffrage Coverage in Republican Newspapers that endorsed suffrage as a \"Republican\" issue:\n")
articles[partisanship %in% "republican", mean(suffrage_republican_issue | suffrage_republican_activist), by = paper_id] %>% 
  .$V1 %>% mean

##############################################################
#Cleanup
rm(list = setdiff(ls(), c(keep, 'keep')))
gc()


